import cv2
import random
from random import sample
import argparse
import numpy as np
import os
import pickle
from tqdm import tqdm
from scipy.interpolate import NearestNDInterpolator
from collections import OrderedDict
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
from sklearn.metrics import precision_recall_curve
from sklearn.covariance import LedoitWolf
from scipy.spatial.distance import mahalanobis
from scipy.ndimage import gaussian_filter
from skimage import morphology
from skimage.segmentation import mark_boundaries
import matplotlib.pyplot as plt
import matplotlib
import cv2 as cv
from PIL import Image
import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader
# Pre-trained model
from torchvision.models import wide_resnet50_2, resnet18 # torchvision library for pre-trained models
# Dataset
import datasets.data_process_original as mvtec # data_process.py is in the datasets folder
import datasets.fft_filter_normalized as fft_filter
C:\Users\kiran\anaconda3\Lib\site-packages\paramiko\transport.py:219: CryptographyDeprecationWarning: Blowfish has been deprecated "class": algorithms.Blowfish,
def plot_fig(test_imgs, scores, fix_ratios, save_dir = "", save = False, title = ""):
"""
test_imgs: : real crack images
scores: test scores for test images
fix_ratio: fix_ratio for mask
save_dir: location of directory for saving the images
"""
class_name = "test " #+ str(fix_ratio)
num = len(test_imgs)
vmax = scores.max() * 255.
vmin = scores.min() * 255.
norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
for i in range(num):
# Plotting
fig_img, ax_img = plt.subplots(4, 4, figsize=(24,26))
fig_img.tight_layout(pad = 0)
fig_img.subplots_adjust(top=0.9)
fig_img.subplots_adjust(right=0.9)
norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
for row in ax_img:
for ax_i in row:
ax_i.axes.xaxis.set_visible(False)
ax_i.axes.yaxis.set_visible(False)
##############################################################
# padim only
img = test_imgs[i]
img = denormalization(img)
for j,fix_ratio in enumerate(fix_ratios):
heat_map = scores[i] * 255
mask = scores[i].copy()
mask_max = mask.max()
mask_min = mask.min()
# threshold
threshold = mask_min + fix_ratio * (mask_max - mask_min)
mask[mask > threshold] = 1
mask[mask <= threshold] = 0
kernel = morphology.disk(4)
mask = morphology.opening(mask, kernel)
mask *= 255
vis_img = mark_boundaries(img, mask, color=(1, 0, 0), mode='thick')
#############################################################
# 1st, main image
ax_img[j,0].imshow(img)
ax_img[j,0].title.set_text('original image')
# 2nd, PADIM Predicted heat map
ax = ax_img[j,1].imshow(heat_map, cmap='jet', norm=norm)
ax_img[j,1].imshow(img, cmap='gray', interpolation='none')
ax_img[j,1].imshow(heat_map, cmap='jet', alpha=0.5, interpolation='none')
ax_img[j,1].title.set_text(f'Predicted heat map with fix_ratio {fix_ratio}')
# 3rd, PADIM Predicted mask
ax_img[j, 2].imshow(mask, cmap='gray')
ax_img[j, 2].title.set_text(f'Predicted mask with fix_ratio {fix_ratio}')
# 4th, PADIM Segmentation result
ax_img[j, 3].imshow(vis_img)
ax_img[j, 3].title.set_text(f'Segmentation result with fix_ratio {fix_ratio}')
left = 0.92
bottom = 0.15
width = 0.015
height = 1 - 2 * bottom
rect = [left, bottom, width, height]
cbar_ax = fig_img.add_axes(rect)
cb = plt.colorbar(ax, shrink=0.6, cax=cbar_ax, fraction=0.046)
cb.ax.tick_params(labelsize=8)
font = {
'family': 'serif',
'color': 'black',
'weight': 'normal',
'size': 8,
}
cb.set_label('Anomaly Score', fontdict=font)
plt.suptitle(title)
if save:
os.makedirs(save_dir + '/compare_imgs/', exist_ok=True)
fig_img.savefig(os.path.join(save_dir + '/compare_imgs/', class_name + str(i)), dpi=100)
plt.close()
else:
plt.show()
plt.close()
break
# device setup
use_cuda = torch.cuda.is_available()
device = torch.device('cuda' if use_cuda else 'cpu')
# name = "transistor"
# # Directories
# save_path = f'./resulting_imgs/data_science/{name}'
# # Creating directory to store the results
# os.makedirs(os.path.join(save_path, 'temp_'+ architecture), exist_ok=True)
# train_feature_filepath = os.path.join(save_path, 'temp_%s' % architecture, 'train_%s.pkl' % class_name)
transistor_path = "transistor/good_misplaced/"
crack_path = "cracks/"
file_location = f'./resulting_imgs/data_science/'
def get_img_embed(path):
file_location = f'./resulting_imgs/data_science/{path}'
train_embedding_vectors = np.load(file_location + "/train_np.npy")
if os.path.exists(file_location + "/test_np.npy"):
test_embedding_vectors = np.load(file_location + "/test_np.npy")
else:
test_embedding_vectors = None
if os.path.exists(file_location + "/test_imgs.npy"):
test_imgs = np.load(file_location + "/test_imgs.npy")
else:
test_imgs = None
bad = np.load(file_location + "/raw_images.npy")
try:
good = np.load(file_location + "/good_imgs.npy")
except:
good = None
return bad, good, train_embedding_vectors, test_embedding_vectors,test_imgs
crack_img ,_ ,crack_embed, _,test_imgs = get_img_embed(crack_path)
bad, good, transistor_embed, tran_test_embed, test_imgs = get_img_embed(transistor_path)
transistor_img.shape
(10, 1024, 1024)
test_imgs.shape
(10, 3, 224, 224)
good.shape
(213, 1024, 1024)
bad.shape
(10, 1024, 1024)
tran_test_embed.shape
(10, 448, 56, 56)
n = 5
bad_img = bad[n]
good_img = good[n]
fig_img, ax_img = plt.subplots(1, 2, figsize=(20, 15))
ax_img[0].imshow(good[n], cmap = "gray")
ax_img[0].title.set_text('Good image')
ax_img[1].imshow( bad[n], cmap = "gray")
ax_img[1].title.set_text('image with defect')
fig_img, ax_img = plt.subplots(1, 2, figsize=(20, 15))
ax_img[0].imshow(fft_filter.fft(torch.tensor(good_img))(25,0), cmap = "gray")
ax_img[0].title.set_text('Good image')
ax_img[1].imshow( fft_filter.fft(torch.tensor(bad_img))(25,0), cmap = "gray")
ax_img[1].title.set_text('image with defect')
fig_img, ax_img = plt.subplots(1, 2, figsize=(20, 15))
ax_img[0].imshow(fft_filter.fft(torch.tensor(good_img))(120,10), cmap = "gray")
ax_img[0].title.set_text('Good image')
ax_img[1].imshow( fft_filter.fft(torch.tensor(bad_img))(120,10), cmap = "gray")
ax_img[1].title.set_text('image with defect')
fig_img, ax_img = plt.subplots(1, 2, figsize=(20, 15))
ax_img[0].imshow(good_img- fft_filter.fft(torch.tensor(good_img))(25,0), cmap = "gray")
ax_img[0].title.set_text('Good image')
ax_img[1].imshow( bad_img - fft_filter.fft(torch.tensor(bad_img))(25,0), cmap = "gray")
ax_img[1].title.set_text('image with defect')
fig_img, ax_img = plt.subplots(1, 2, figsize=(20, 15))
ax_img[0].imshow(good_img- fft_filter.fft(torch.tensor(good_img))(120,30), cmap = "gray")
ax_img[0].title.set_text('Good image')
ax_img[1].imshow( bad_img - fft_filter.fft(torch.tensor(bad_img))(120,30), cmap = "gray")
ax_img[1].title.set_text('image with defect')
n = 5
img = crack_img[n][0]
img = torch.tensor(img)
plt.imshow(img, cmap = "gray")
<matplotlib.image.AxesImage at 0x20e9a654090>
fft_img = fft_filter.fft(img)(120,20)
plt.imshow(fft_img,cmap="gray")
<matplotlib.image.AxesImage at 0x20e98ed1f50>
transistor_embed.shape
(213, 448, 56, 56)
idx = random.sample(range(448), 9)
figure, axis = plt.subplots(3, 3 , figsize=(16, 16))
figure.tight_layout(pad = 0)
for i in range(3):
for j in range(3):
img_num = 0
channel_num = idx[3*i+j]
channel = transistor_embed[img_num][channel_num]
axis[i,j].imshow(channel, cmap = "gray")
axis[i,j].set_title(f"embedding_vectors[{img_num}][{channel_num}]" )
plt.show()
The initial channels are from first layer. So, it depicts low level of the image
figure, axis = plt.subplots(3, 3 , figsize=(16, 16))
figure.tight_layout(pad = 0)
for i in range(3):
for j in range(3):
img_num = 0
channel_num = 0
channel = transistor_embed[img_num][channel_num + 3*i + j]
axis[i,j].imshow(channel, cmap = "gray")
axis[i,j].set_title(f"embedding_vectors[{img_num}][{channel_num}]" )
plt.show()
figure, axis = plt.subplots(3, 3 , figsize=(16, 16))
figure.tight_layout(pad = 0)
for i in range(3):
for j in range(3):
img_num = 0
channel_num = 50
channel = transistor_embed[img_num][channel_num + 3*i + j]
axis[i,j].imshow(channel, cmap = "gray")
axis[i,j].set_title(f"embedding_vectors[{img_num}][{channel_num}]" )
plt.show()
figure, axis = plt.subplots(3, 3 , figsize=(16, 16))
figure.tight_layout(pad = 0)
for i in range(3):
for j in range(3):
img_num = 0
channel_num = 100
channel = transistor_embed[img_num][channel_num + 3*i + j]
axis[i,j].imshow(channel, cmap = "gray")
axis[i,j].set_title(f"embedding_vectors[{img_num}][{channel_num}]" )
plt.show()
figure, axis = plt.subplots(3, 3 , figsize=(16, 16))
figure.tight_layout(pad = 0)
for i in range(3):
for j in range(3):
img_num = 0
channel_num = 250
channel = transistor_embed[img_num][channel_num + 3*i + j]
axis[i,j].imshow(channel, cmap = "gray")
axis[i,j].set_title(f"embedding_vectors[{img_num}][{channel_num}]" )
plt.show()
figure, axis = plt.subplots(3, 3 , figsize=(16, 16))
figure.tight_layout(pad = 0)
for i in range(3):
for j in range(3):
img_num = 0
channel_num = 430
channel = transistor_embed[img_num][channel_num + 3*i + j]
axis[i,j].imshow(channel, cmap = "gray")
axis[i,j].set_title(f"embedding_vectors[{img_num}][{channel_num + 3*i + j}]" )
plt.show()
figure, axis = plt.subplots(2, 3 , figsize=(16, 12))
figure.tight_layout(pad = 0)
nums = [1, 3]
for i in range(2):
k = 0
for j in range(3):
img_num = nums[i]
channel_num = 60
channel = transistor_embed[img_num][channel_num + 3*k + j]
axis[i,j].imshow(channel, cmap = "gray")
axis[i,j].set_title(f"embedding_vectors[{img_num}][{channel_num + 3*k + j}]" )
plt.show()
figure, axis = plt.subplots(2, 3 , figsize=(16, 12))
figure.tight_layout(pad = 0)
nums = [1, 3]
for i in range(2):
k = 0
for j in range(3):
img_num = nums[i]
channel_num = 400
channel = transistor_embed[img_num][channel_num + 3*k + j]
axis[i,j].imshow(channel, cmap = "gray")
axis[i,j].set_title(f"embedding_vectors[{img_num}][{channel_num + 3*k + j}]" )
plt.show()
They are kind of similar
# Distance betweeb channel and its fft with raidus 20, 0
def calculateDistance(c):
channel = c.copy()
obj = fft_filter.fft(torch.from_numpy(channel))
c_fft = obj(20,0)
if np.isnan(c_fft).all():
return 0.00
return ((channel-c_fft)**2).sum()
def filter_channels(embedding_vectors, a, b):
embed_np = embedding_vectors.numpy().copy()
_imgs = []
# dictionary for each image with all its channels
for img_id, image in enumerate(embed_np):
# stores all the distances of channel and its fft
channel_dist = {}
channel_dict = {}
for channel_id, channel in enumerate(image):
channel_dist[channel_id] = calculateDistance(channel)
channel_dict[channel_id] = channel
# Sort the dictionary
channel_dist = dict(sorted(channel_dist.items(), key=lambda item: item[1]))
# select top 100 channels and replace it in the original dictonary
channel_dict = {k: channel_dict[k] for k in list(channel_dist.keys())[a:b]}
# append to the list
_imgs.append(channel_dict)
return _imgs
sorted_embed = filter_channels(torch.tensor(transistor_embed), 0, 448)
C:\Users\kiran\OneDrive - Tulane University\semesters\2022 Fall\cv\codes\Segmentation\Segmentation\datasets\fft_filter_normalized.py:106: RuntimeWarning: invalid value encountered in divide img = img / (_max - _min) C:\Users\kiran\OneDrive - Tulane University\semesters\2022 Fall\cv\codes\Segmentation\Segmentation\datasets\fft_filter_normalized.py:41: RuntimeWarning: invalid value encountered in cast img_mandp = np.uint8(np.abs(np.fft.ifft2(img_mandp))) C:\Users\kiran\OneDrive - Tulane University\semesters\2022 Fall\cv\codes\Segmentation\Segmentation\datasets\fft_filter_normalized.py:44: RuntimeWarning: invalid value encountered in divide img_mandp=img_mandp/np.max(img_mandp)*255
sorted_embed[0][272].shape
(56, 56)
img_idx = [0 ,2, 5]
scatter_list = {}
for i in img_idx:
order = sorted_embed[i].keys()
scatter_list[i] = order
indices = list(range(0,448,10))
img0_order = [ list(scatter_list[0])[i] for i in indices]
img2_order = [ list(scatter_list[2])[i] for i in indices]
img5_order = [ list(scatter_list[5])[i] for i in indices]
fig, ax = plt.subplots(figsize = (16,8))
plt.scatter(indices, img0_order)
plt.scatter(indices, img2_order)
plt.scatter(indices, img5_order)
plt.xlabel("Order of channels after sorting")
plt.ylabel("Indices of the channels ")
plt.legend(["img 0", "img 2", "img 5"])
plt.title("Ordering of channels for different images")
# Enabling minor grid lines:
ax.grid(which = "both")
ax.minorticks_on()
plt.show()
indices = list(range(0,448,20))
img0_order = [ list(scatter_list[0])[i] for i in indices]
img2_order = [ list(scatter_list[2])[i] for i in indices]
img5_order = [ list(scatter_list[5])[i] for i in indices]
plt.figure(figsize = (16,8))
plt.plot(indices, img0_order)
plt.plot(indices, img2_order)
plt.plot(indices, img5_order)
plt.xlabel("Order of channels after sorting")
plt.ylabel("Indices of the channels ")
plt.legend(["img 0", "img 2", "img 5"])
plt.title("Ordering of channels for different images")
plt.legend(["img 0", "img 2", "img 5"])
plt.grid()
plt.show()
Hence it shows that FFT distance does not order uniformly
def filter_channels_using_oneImg(embedding_vectors, a, b, rand_int):
embed_np = transistor_embed.copy()
_imgs = []
img_id = rand_int
image = embed_np[img_id]
# stores all the distances of channel and its fft
channel_dist = {}
channel_dict = {}
for channel_id, channel in enumerate(image):
channel_dist[channel_id] = calculateDistance(channel)
channel_dict[channel_id] = channel
# Sort the dictionary
channel_dist = dict(sorted(channel_dist.items(), key=lambda item: item[1]))
# we take the index of this dictionary and seelct the top ones
# Based on the index of the top ones, we slice the dictionary
return torch.index_select(torch.tensor(embed_np), 1, torch.tensor(list(channel_dist.keys())[a:b])), torch.tensor(list(channel_dist.keys())[a:b])
img_id = random.randint(0,embed_np.shape[0])
low_sorted_embed, low_order = filter_channels_using_oneImg(torch.tensor(transistor_embed), 0, 100, img_id)
high_sorted_embed, high_order = filter_channels_using_oneImg(torch.tensor(transistor_embed), 300, 400, img_id)
mid_sorted_embed, mid_order = filter_channels_using_oneImg(torch.tensor(transistor_embed), 100, 300, img_id)
This means for all the images, we have 272 th indexed channel in the first place after slicing
img_idx = [0 ,2, 5]
scatter_list = {}
for i in img_idx:
order = sorted_embed[i].keys()
scatter_list[i] = order
# order_indices = list(range(0,448,10))
# img0_order = [ list(low_order[i])[i] for i in list(range(0,100))]
# img2_order = [ list(high_order[i])[i] for i in list(range(300,400))]
# img5_order = [ list(mid_order[i] for i in list(range(100,300)))]
fig, ax = plt.subplots(figsize = (16,8))
plt.scatter(list(range(0,100)), low_order)
plt.scatter(list(range(300,400)), high_order)
plt.scatter(list(range(100,300)), mid_order)
plt.xlabel("Order of channels after sorting")
plt.ylabel("Indices of the channels ")
plt.legend(["low", "high", "mid"])
plt.title("Ordering of channels for the same image")
# Enabling minor grid lines:
ax.grid(which = "both")
ax.minorticks_on()
plt.show()
Not much information here
tran_test_embed.shape
(10, 448, 56, 56)
transistor_embed.shape
(213, 448, 56, 56)
low_sorted_embed.shape
torch.Size([213, 100, 56, 56])
torch.tensor(transistor_embed).shape
torch.Size([213, 448, 56, 56])
train_outputs = []
for x_embed in [low_sorted_embed, mid_sorted_embed, high_sorted_embed]:
B, C, H, W = x_embed.shape
embedding_vectors = x_embed.view(B, C, H * W)
mean = torch.mean(embedding_vectors, dim=0).numpy()
# shape C, C,3136
cov = torch.zeros(C, C, H * W).numpy()
I = np.identity(C)
# Covariance for each feature index
for i in range(H * W):
# Each C x C from C x C x (H * W)
cov[:, :, i] = np.cov(embedding_vectors[:, :, i].numpy(), rowvar=False) + 0.01 * I
train_outputs.append([mean, cov])
train_outputs[0][0].shape
(100, 3136)
len(train_outputs)
3
tran_test_embed.shape
(10, 448, 56, 56)
scores = []
for k, _order in enumerate([low_order, mid_order, high_order]):
embedding_vectors = torch.index_select(torch.tensor(tran_test_embed), 1, _order)
# calculate distance matrix
B, C, H, W = embedding_vectors.size()
embedding_vectors = embedding_vectors.view(B, C, H * W).numpy()
dist_list = []
for i in range(H * W):
# ith feature mean, shape (100,)
# Each specific attrribute is represented by 100 x 1 mean vector
mean = train_outputs[k][0][:, i]
# Inverse of covariance of ith feature
conv_inv = np.linalg.inv(train_outputs[k][1][:, :, i])
"""
_img: 100 channels and 3136 feature indices, shape = (100, 3136) for particular image
_img[:, i] shape (100,); for ith feature gives all 100 channel values
"""
dist = [mahalanobis(_img[:, i], mean, conv_inv) for _img in embedding_vectors]
dist_list.append(dist)
# Reshaping to 449 x 56 x 56
dist_list = np.array(dist_list).transpose(1, 0).reshape(B, H, W)
# Numpy into tensor
dist_list = torch.tensor(dist_list)
# unsqueeze adds dimenstion and new dimension is (449, 1, 56, 56)
# size is 224, which is size of original img
# squeeze() removes the single dimensions
score_map = F.interpolate(dist_list.unsqueeze(1), size= 224, mode='bilinear', align_corners=False).squeeze().numpy()
# apply gaussian smoothing on the score map (on each 56 x 56 imgs)
for i in range(score_map.shape[0]):
score_map[i] = gaussian_filter(score_map[i], sigma=4)
# Normalization
max_score = score_map.max()
min_score = score_map.min()
_scores = (score_map - min_score) / (max_score - min_score)
scores.append(_scores)
def denormalization(x):
mean = np.array([0.485, 0.456, 0.406])
std = np.array([0.229, 0.224, 0.225])
x = (((x.transpose(1, 2, 0) * std) + mean) * 255.).astype(np.uint8)
return x
save = False
fix_ratios = [0.5, 0.6, 0.65, 0.7]
plot_fig(test_imgs ,scores[0], fix_ratios, save_dir, title = orders[j])
save = False
fix_ratios = [0.5, 0.6, 0.65, 0.7]
orders = ["low", "mid", "high"]
for j in range(3):
print(orders[j])
plot_fig(test_imgs ,scores[j], fix_ratios, save_dir, title = orders[j])
low
mid
high
extra = "mid_Order"
save_dir = file_location + transistor_path +"results/"+ f'pictures' + "_"+ extra
save_dir
'./resulting_imgs/data_science/transistor/good_misplaced/results/pictures_mid_Order'
os.makedirs(save_dir, exist_ok=True)
fix_ratios = [0.5, 0.6, 0.65, 0.7]
plot_fig(test_imgs ,scores, fix_ratios, save_dir)